from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
# import gc
import copy
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import re
import numpy as np
from os import walk
# import matplotlib.pyplot as plt
from subprocess import call
import os

def step3():
    pass

def step2():
    print "Begins..."
    totalNumOfPostings = 6451948010
    onePercentOfTotalNumOfPostings = totalNumOfPostings * 0.01
    twoPercentOfTotalNumOfPostings = totalNumOfPostings * 0.02
    fivePercentOfTotalNumOfPostings = totalNumOfPostings * 0.05
    tenPercentOfTotalNumOfPostings = totalNumOfPostings * 0.1
    twentyPercentOfTotalNumOfPostings = totalNumOfPostings * 0.2
    thirtyPercentOfTotalNumOfPostings = totalNumOfPostings * 0.3
    fortyPercentOfTotalNumOfPostings = totalNumOfPostings * 0.4
    fiftyPercentOfTotalNumOfPostings = totalNumOfPostings * 0.5
    
    numOfPostingsList = []
    numOfPostingsList.append(onePercentOfTotalNumOfPostings)
    numOfPostingsList.append(twoPercentOfTotalNumOfPostings)
    numOfPostingsList.append(fivePercentOfTotalNumOfPostings)
    numOfPostingsList.append(tenPercentOfTotalNumOfPostings)
    numOfPostingsList.append(twentyPercentOfTotalNumOfPostings)
    numOfPostingsList.append(thirtyPercentOfTotalNumOfPostings)
    numOfPostingsList.append(fortyPercentOfTotalNumOfPostings)
    numOfPostingsList.append(fiftyPercentOfTotalNumOfPostings)

    '''
    ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_point2MQueries_final_sortedByHitFreq_withNumOfPostingsAdded"
    ifn1 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_point3MQueries_final_sortedByHitFreq_withNumOfPostingsAdded"
    ifn2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_1MQueries_final_sortedByHitFreq_withNumOfPostingsAdded"
    ifn3 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_2MQueries_final_sortedByHitFreq_withNumOfPostingsAdded"
    ifn4 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_5MQueries_final_sortedByHitFreq_withNumOfPostingsAdded"
    ifn5 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_10MQueries_final_sortedByHitFreq_withNumOfPostingsAdded"
    ifn6 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_20MQueries_final_sortedByHitFreq_withNumOfPostingsAdded"
    '''
    ifn7 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_from50M/resultfile0_50MQueries_final_sortedByHitFreq_withNumOfPostingsAdded"

    '''
    ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_point2MQueries_final_withNumOfPostingsAdded_sortedByDocHitsDividedByDocSize"
    ifn1 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_point3MQueries_final_withNumOfPostingsAdded_sortedByDocHitsDividedByDocSize"
    ifn2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_1MQueries_final_withNumOfPostingsAdded_sortedByDocHitsDividedByDocSize"
    ifn3 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_2MQueries_final_withNumOfPostingsAdded_sortedByDocHitsDividedByDocSize"
    ifn4 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_5MQueries_final_withNumOfPostingsAdded_sortedByDocHitsDividedByDocSize"
    ifn5 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_10MQueries_final_withNumOfPostingsAdded_sortedByDocHitsDividedByDocSize"
    ifn6 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_20MQueries_final_withNumOfPostingsAdded_sortedByDocHitsDividedByDocSize"
    ifn7 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_50MQueries_final_withNumOfPostingsAdded_sortedByDocHitsDividedByDocSize"
    '''

    '''
    ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_10MQueries_final_sortedByHitFreq_withNumOfPostingsAdded"
    ifn1 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_10MQueries_previous_final_sortedByHitFreq_withNumOfPostingsAdded"
    '''
 
    '''
    ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_point2MQueries_final_withNumOfPostingsAdded_sortedByDocHitsDividedBySQRTDocSize"
    ifn1 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_point3MQueries_final_withNumOfPostingsAdded_sortedByDocHitsDividedBySQRTDocSize"
    ifn2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_1MQueries_final_withNumOfPostingsAdded_sortedByDocHitsDividedBySQRTDocSize"
    ifn3 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_2MQueries_final_withNumOfPostingsAdded_sortedByDocHitsDividedBySQRTDocSize"
    ifn4 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_5MQueries_final_withNumOfPostingsAdded_sortedByDocHitsDividedBySQRTDocSize"
    ifn5 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_10MQueries_final_withNumOfPostingsAdded_sortedByDocHitsDividedBySQRTDocSize"
    ifn6 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_20MQueries_final_withNumOfPostingsAdded_sortedByDocHitsDividedBySQRTDocSize"
    ifn7 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_50MQueries_final_withNumOfPostingsAdded_sortedByDocHitsDividedBySQRTDocSize"
    '''

    fileList = []
    '''
    fileList.append(ifn0)
    fileList.append(ifn1)
    fileList.append(ifn2)
    fileList.append(ifn3)
    fileList.append(ifn4)
    fileList.append(ifn5)
    fileList.append(ifn6)
    '''
    fileList.append(ifn7)
    
    '''
    fileList.append(ifn0)
    fileList.append(ifn1)
    fileList.append(ifn2)
    fileList.append(ifn3)
    fileList.append(ifn4)
    fileList.append(ifn5)
    fileList.append(ifn6)
    fileList.append(ifn7)
    '''

    for currentNumOfPostingValue in numOfPostingsList:
	    for fileName in fileList:
		tier1DocDict = {}
		ifh0 = open(fileName,"r")
		l = ifh0.readline()
		sumOfPostings = 0
		while l:
		    le = l.strip().split(" ")
		    docID = le[0]
		    currHitFreq = int(le[1])
		    sumOfPostings += int(le[2])
		    if sumOfPostings > currentNumOfPostingValue:
			break
		    tier1DocDict[docID] = 1
		    l = ifh0.readline()
		# print "Overall:"
		# print "len(tier1DocDict):",len(tier1DocDict)
		print len(tier1DocDict),sumOfPostings,currHitFreq
		ifh0.close()

		'''
		# step2: compute the OVERLAP, ignore from now
		counter = 0
		ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/TOP10_documentResults_OR_unpruned_testingQueries_20141020"
		ifh = open(ifn,"r")
		l = ifh.readline()
		while l:
		    le = l.strip().split(" ")
		    docID = le[3]
		    if docID in tier1DocDict:
			counter += 1
		    l = ifh.readline()
		ifh.close()
		
		print counter,counter/49988
		#print "Overall:"
		#print "Overlap quality:",counter,counter/49988
		#print "End."
		'''

def step1():
    print "Begins..."
    docIDAndNumOfPostingsDict = {}
    ifn = "/home/vgc/wei/workspace/NYU_IRTK/data/gov2_docID_trecID_numOfPostings"
    ifh = open(ifn,"r")
    l = ifh.readline()
    counter = 1
    while l:
        le = l.strip().split(" ")
        docID = le[0]
        trecID = le[1]
        numOfPostings = int(le[2])
        docIDAndNumOfPostingsDict[docID] = numOfPostings 
        l = ifh.readline()
        counter += 1
        #if counter % 1000 == 0:
        #    print counter
    print "Overall:"
    print "len(docIDAndNumOfPostingsDict):",len(docIDAndNumOfPostingsDict)
    print "docIDAndNumOfPostingsDict['0']:",docIDAndNumOfPostingsDict['0']
    ifh.close()
    
    '''
    ifn1 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_point2MQueries_final_sortedByHitFreq"
    ifn2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_point3MQueries_final_sortedByHitFreq"
    ifn3 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_1MQueries_final_sortedByHitFreq"
    ifn4 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_2MQueries_final_sortedByHitFreq"
    ifn5 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_5MQueries_final_sortedByHitFreq"
    ifn6 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_10MQueries_final_sortedByHitFreq"
    ifn7 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_20MQueries_final_sortedByHitFreq"
    ifn8 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_50MQueries_final_sortedByHitFreq"
    '''
    ifn9 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/docHits_20141030/resultfile0_10MQueries_previous_final_sortedByHitFreq"
    fileList = []
    '''
    fileList.append(ifn1)
    fileList.append(ifn2)
    fileList.append(ifn3)
    fileList.append(ifn4)
    fileList.append(ifn5)
    fileList.append(ifn6)
    fileList.append(ifn7)
    fileList.append(ifn8)
    '''
    fileList.append(ifn9)
    for fileName in fileList:
        ifh = open(fileName,"r")
        ofn = fileName + "_withNumOfPostingsAdded"
        ofh = open(ofn,"w")
        l = ifh.readline()
        while l:
            le = l.strip().split(" ")
            docID = le[0]
            numOfPostings = docIDAndNumOfPostingsDict[docID]
            ofh.write(docID + " " + le[1] + " " + str(numOfPostings) + "\n")
            l = ifh.readline()
        print "Overall:"
        print "fileName:",fileName
        print "ofn:",ofn
        ifh.close()
        ofh.close()
    print "Ends."

def step4():    
    counter = 0
    unprunedTOPResultDict = {}
    ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/TOP10_documentResults_OR_unpruned_testingQueries_20141020"
    ifh = open(ifn,"r")
    l = ifh.readline()
    while l:
        le = l.strip().split(" ")
        queryID = le[0]
        docID = le[3]
        documentResult = queryID + "_" + docID
        unprunedTOPResultDict[documentResult] = 1
        counter += 1
        l = ifh.readline()
    ifh.close()
    print "Overall:"
    print "len(unprunedTOPResultDict):",len(unprunedTOPResultDict)
    print "counter:",counter
    
    thatResultNumber = 0
    # ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/rawResults_max-score_postingHit_20141107_attempt3"
    ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/rawResults_max-score_postingHit_20141109_2Point7Percent"
    ifh = open(ifn,"r")
    l = ifh.readline()
    while l:
        le = l.strip().split(" ")
        #print le
        #print len(le)
        
        if l.strip().startswith("qid:"):
            queryID = le[1]
        if len(le) == 35 and le[-2].strip().startswith("GX"):
            docID = le[-3]
            documentResult = queryID + "_" + docID
            if documentResult in unprunedTOPResultDict:
                thatResultNumber += 1  
        
        l = ifh.readline()
    
    ifh.close()
    print "Overall:"
    print "thatResultNumber:",thatResultNumber

print "Begins..."
#step1() # do the aux file preparation
step2() # do the overlap quality for the docHits methods
#step3() # do the P@10 quality
#step4() # do the overlap quality for the postingHits methods
print "Ends."


